{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 6862,
     "status": "ok",
     "timestamp": 1650010812842,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "98nP9Uh9GUTL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.0, 4.344421863555908, 4.2579545974731445]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([0.10000000149011612,\n",
       "  0.20000000298023224,\n",
       "  4.344421863555908,\n",
       "  4.2579545974731445],\n",
       " -1.0,\n",
       " False)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import random\n",
    "\n",
    "random.seed(0)\n",
    "torch.manual_seed(0)\n",
    "\n",
    "#创建一个游戏环境\n",
    "class Env:\n",
    "    def reset(self):\n",
    "        #前两个数是起点,后两个数是终点\n",
    "        self.state = torch.zeros(4)\n",
    "        self.state[2] = random.uniform(3.5, 4.5)\n",
    "        self.state[3] = random.uniform(3.5, 4.5)\n",
    "\n",
    "        self.count = 0\n",
    "\n",
    "        return self.state.tolist()\n",
    "\n",
    "    def step(self, action):\n",
    "        action = torch.FloatTensor(action).reshape(2)\n",
    "\n",
    "        #裁剪动作范围\n",
    "        action = torch.clamp(action, min=-1, max=1)\n",
    "\n",
    "        #执行动作\n",
    "        self.state[:2] += action\n",
    "\n",
    "        #规范状态空间\n",
    "        self.state[:2] = torch.clamp(self.state[:2], min=0, max=5)\n",
    "\n",
    "        self.count += 1\n",
    "\n",
    "        #求二范数,求两向量相减之后的向量的模长\n",
    "        #两向量相减的几何意义是两个向量的尾部相连,再连接两个头部形成的新向量\n",
    "        #mod = ((self.state - self.goal)**2).sum()**0.5\n",
    "        mod = (self.state[:2] - self.state[2:]).norm(p=2).item()\n",
    "\n",
    "        reward = -1.0\n",
    "        over = False\n",
    "        if mod <= 0.15:\n",
    "            reward = 0.0\n",
    "            over = True\n",
    "\n",
    "        if self.count >= 50:\n",
    "            over = True\n",
    "\n",
    "        return self.state.tolist(), reward, over\n",
    "\n",
    "\n",
    "env = Env()\n",
    "\n",
    "print(env.reset())\n",
    "\n",
    "env.step([0.1, 0.2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1650010819329,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "bxiqOl_vGUTR"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-0.3392077088356018, 0.055921927094459534]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class DDPG:\n",
    "    def __init__(self):\n",
    "        self.model_action = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 2),\n",
    "            torch.nn.Tanh(),\n",
    "        )\n",
    "        self.model_value = torch.nn.Sequential(\n",
    "            torch.nn.Linear(6, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "        )\n",
    "\n",
    "        self.model_action_next = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 2),\n",
    "            torch.nn.Tanh(),\n",
    "        )\n",
    "        self.model_value_next = torch.nn.Sequential(\n",
    "            torch.nn.Linear(6, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 1),\n",
    "        )\n",
    "\n",
    "        self.model_action_next.load_state_dict(self.model_action.state_dict())\n",
    "        self.model_value_next.load_state_dict(self.model_value.state_dict())\n",
    "\n",
    "        self.optimizer_action = torch.optim.Adam(\n",
    "            self.model_action.parameters(), lr=1e-3)\n",
    "        self.optimizer_value = torch.optim.Adam(self.model_value.parameters(),\n",
    "                                                lr=1e-3)\n",
    "\n",
    "        self.mse_loss = torch.nn.MSELoss()\n",
    "\n",
    "    def get_action(self, state):\n",
    "        state = torch.FloatTensor(state).reshape(1, 4)\n",
    "\n",
    "        #[1, 4] -> [1, 2]\n",
    "        action = self.model_action(state)\n",
    "\n",
    "        #[1, 2] -> [2]\n",
    "        action = action.reshape(2)\n",
    "\n",
    "        #添加噪声,增加探索\n",
    "        action += 0.1 * torch.randn(2)\n",
    "        return action.tolist()\n",
    "\n",
    "    def _soft_update(self, model, model_next):\n",
    "        for param, param_next in zip(model.parameters(),\n",
    "                                     model_next.parameters()):\n",
    "            #以一个小的比例更新\n",
    "            value = param_next.data * 0.995 + param.data * 0.005\n",
    "            param_next.data.copy_(value)\n",
    "\n",
    "    def _get_target(self, next_state, reward, over):\n",
    "        #[b, 4] -> [b, 2]\n",
    "        action = self.model_action_next(next_state)\n",
    "\n",
    "        #[b, 4+2] -> [b, 6]\n",
    "        input = torch.cat([next_state, action], dim=1)\n",
    "\n",
    "        #[b, 6] -> [b, 1]\n",
    "        target = self.model_value_next(input)\n",
    "\n",
    "        #[b, 1]\n",
    "        target *= 0.98\n",
    "        target *= (1 - over)\n",
    "        target += reward\n",
    "\n",
    "        return target\n",
    "\n",
    "    def _get_value(self, state, action):\n",
    "        #[b, 4+2] -> [b, 6]\n",
    "        input = torch.cat([state, action], dim=1)\n",
    "\n",
    "        #[b, 6] -> [b, 1]\n",
    "        value = self.model_value(input)\n",
    "\n",
    "        return value\n",
    "\n",
    "    def _get_loss_action(self, state):\n",
    "        #[b, 4] -> [b, 2]\n",
    "        action = self.model_action(state)\n",
    "\n",
    "        #[b, 4+2] -> [b, 6]\n",
    "        input = torch.cat([state, action], dim=1)\n",
    "\n",
    "        #[b, 6] -> [b, 1]\n",
    "        loss_action = self.model_value(input)\n",
    "        loss_action = -loss_action.mean()\n",
    "\n",
    "        return loss_action\n",
    "\n",
    "    def train(self, state, action, reward, next_state, over):\n",
    "        #state -> [b, 4]\n",
    "        #action -> [b, 2]\n",
    "        #reward -> [b, 1]\n",
    "        #next_state -> [b, 4]\n",
    "        #over -> [b, 1]\n",
    "\n",
    "        #[b, 1]\n",
    "        target = self._get_target(next_state, reward, over)\n",
    "\n",
    "        #[b, 1]\n",
    "        value = self._get_value(state, action)\n",
    "\n",
    "        loss_value = self.mse_loss(value, target)\n",
    "        self.optimizer_value.zero_grad()\n",
    "        loss_value.backward()\n",
    "        self.optimizer_value.step()\n",
    "\n",
    "        loss_action = self._get_loss_action(state)\n",
    "        self.optimizer_action.zero_grad()\n",
    "        loss_action.backward()\n",
    "        self.optimizer_action.step()\n",
    "\n",
    "        self._soft_update(self.model_action, self.model_action_next)\n",
    "        self._soft_update(self.model_value, self.model_value_next)\n",
    "\n",
    "\n",
    "ddpg = DDPG()\n",
    "\n",
    "ddpg.train(\n",
    "    torch.randn(200, 4),\n",
    "    torch.randn(200, 2),\n",
    "    torch.randn(200, 1),\n",
    "    torch.randn(200, 4),\n",
    "    torch.zeros(200, 1).long(),\n",
    ")\n",
    "\n",
    "ddpg.get_action([1, 2, 3, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'state': tensor([[0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0325],\n",
       "          ...,\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 3.9886, 4.0849]]),\n",
       "  'action': tensor([[-0.2635, -0.0956],\n",
       "          [-0.3706, -0.1727],\n",
       "          [-0.4140, -0.1666],\n",
       "          [-0.4562, -0.1237],\n",
       "          [-0.4448, -0.3142],\n",
       "          [-0.3906, -0.1910],\n",
       "          [-0.4076, -0.1296],\n",
       "          [-0.2947, -0.1786],\n",
       "          [-0.3101, -0.2688],\n",
       "          [-0.3959, -0.1792],\n",
       "          [-0.3529,  0.0933],\n",
       "          [-0.5559, -0.0750],\n",
       "          [-0.2379, -0.1012],\n",
       "          [-0.2405, -0.1720],\n",
       "          [-0.2626, -0.1283],\n",
       "          [-0.3308, -0.2549],\n",
       "          [-0.5064, -0.1311],\n",
       "          [-0.3473, -0.1086],\n",
       "          [-0.1949, -0.1427],\n",
       "          [-0.3029, -0.0756],\n",
       "          [-0.3205, -0.0882],\n",
       "          [-0.3693,  0.0324],\n",
       "          [-0.3083, -0.2556],\n",
       "          [-0.3472, -0.3636],\n",
       "          [-0.3210, -0.1942],\n",
       "          [-0.4101, -0.1267],\n",
       "          [-0.1465, -0.1570],\n",
       "          [-0.3262,  0.0079],\n",
       "          [-0.2790, -0.0267],\n",
       "          [-0.2388, -0.1106],\n",
       "          [-0.2410, -0.1997],\n",
       "          [-0.2524, -0.1004],\n",
       "          [-0.4101, -0.1761],\n",
       "          [-0.1523, -0.1412],\n",
       "          [-0.2988, -0.1124],\n",
       "          [-0.4659, -0.0781],\n",
       "          [-0.3869, -0.1591],\n",
       "          [-0.3202, -0.1870],\n",
       "          [-0.2957, -0.2251],\n",
       "          [-0.4045,  0.0503],\n",
       "          [-0.2431, -0.1926],\n",
       "          [-0.3871, -0.2253],\n",
       "          [-0.2799, -0.0436],\n",
       "          [-0.2418, -0.3926],\n",
       "          [-0.3193, -0.0761],\n",
       "          [-0.1482, -0.1245],\n",
       "          [-0.3859, -0.1594],\n",
       "          [-0.1336, -0.1303],\n",
       "          [-0.2871, -0.0375],\n",
       "          [-0.3061, -0.0960],\n",
       "          [-0.3149, -0.1166],\n",
       "          [-0.2313, -0.0381],\n",
       "          [-0.2330, -0.1768],\n",
       "          [-0.3588, -0.1910],\n",
       "          [-0.3092, -0.0578],\n",
       "          [-0.3724, -0.1674],\n",
       "          [-0.4365, -0.1500],\n",
       "          [-0.3093, -0.0530],\n",
       "          [-0.1653, -0.0059],\n",
       "          [-0.3688, -0.1060],\n",
       "          [-0.4076, -0.0930],\n",
       "          [-0.3292, -0.2370],\n",
       "          [-0.2363, -0.2237],\n",
       "          [-0.3431, -0.0502],\n",
       "          [-0.4406, -0.0934],\n",
       "          [-0.2216,  0.0927],\n",
       "          [-0.3363, -0.1693],\n",
       "          [-0.2150, -0.0019],\n",
       "          [-0.2931, -0.1517],\n",
       "          [-0.4154, -0.1849],\n",
       "          [-0.3268, -0.0796],\n",
       "          [-0.2399,  0.0752],\n",
       "          [-0.3949, -0.1679],\n",
       "          [-0.3455, -0.0912],\n",
       "          [-0.2904, -0.2783],\n",
       "          [-0.3263, -0.0918],\n",
       "          [-0.3835,  0.0507],\n",
       "          [-0.4571, -0.1737],\n",
       "          [-0.2682, -0.0294],\n",
       "          [-0.3008,  0.1470],\n",
       "          [-0.2030, -0.0257],\n",
       "          [-0.1567, -0.1672],\n",
       "          [-0.2973, -0.0515],\n",
       "          [-0.2982, -0.1079],\n",
       "          [-0.3261, -0.1670],\n",
       "          [-0.3047, -0.0968],\n",
       "          [-0.1562, -0.0258],\n",
       "          [-0.2910, -0.1669],\n",
       "          [-0.2874, -0.0629],\n",
       "          [-0.4040, -0.0911],\n",
       "          [-0.3176, -0.0237],\n",
       "          [-0.4585, -0.0666],\n",
       "          [-0.2818, -0.0537],\n",
       "          [-0.3915, -0.2092],\n",
       "          [-0.4107, -0.1044],\n",
       "          [-0.2868,  0.0395],\n",
       "          [-0.2541, -0.1807],\n",
       "          [-0.4400, -0.1035],\n",
       "          [-0.2095, -0.1904],\n",
       "          [-0.3411,  0.0075],\n",
       "          [-0.2931, -0.1875],\n",
       "          [-0.2585, -0.2378],\n",
       "          [-0.2346,  0.2285],\n",
       "          [-0.2041, -0.2161],\n",
       "          [-0.2791,  0.0260],\n",
       "          [-0.2417, -0.1279],\n",
       "          [-0.2889,  0.0289],\n",
       "          [-0.2865, -0.0378],\n",
       "          [-0.2548, -0.0104],\n",
       "          [-0.2277, -0.0500],\n",
       "          [-0.3147, -0.1660],\n",
       "          [-0.0680, -0.0907],\n",
       "          [-0.2561, -0.0999],\n",
       "          [-0.3792, -0.2778],\n",
       "          [-0.3498, -0.2610],\n",
       "          [-0.4009, -0.1091],\n",
       "          [-0.2494,  0.0969],\n",
       "          [-0.3605, -0.0917],\n",
       "          [-0.1149, -0.1604],\n",
       "          [-0.4003, -0.0780],\n",
       "          [-0.2033, -0.1403],\n",
       "          [-0.1955, -0.1801],\n",
       "          [-0.1080, -0.2482],\n",
       "          [-0.2153,  0.0757],\n",
       "          [-0.3378, -0.2348],\n",
       "          [-0.3588, -0.1974],\n",
       "          [-0.2322, -0.1390],\n",
       "          [-0.3422, -0.1657],\n",
       "          [-0.4195, -0.2874],\n",
       "          [-0.2897, -0.0248],\n",
       "          [-0.3486, -0.2837],\n",
       "          [-0.1460, -0.1558],\n",
       "          [-0.2956, -0.1122],\n",
       "          [-0.3410, -0.1182],\n",
       "          [-0.3789, -0.2748],\n",
       "          [-0.4616, -0.0696],\n",
       "          [-0.4060, -0.2198],\n",
       "          [-0.4202, -0.0087],\n",
       "          [-0.2002, -0.1933],\n",
       "          [-0.1968, -0.1270],\n",
       "          [-0.2824, -0.2464],\n",
       "          [-0.3903, -0.1241],\n",
       "          [-0.1571, -0.2860],\n",
       "          [-0.2879, -0.0428],\n",
       "          [-0.2615, -0.1140],\n",
       "          [-0.3663, -0.1314],\n",
       "          [-0.3361,  0.1123],\n",
       "          [-0.1883, -0.3343],\n",
       "          [-0.4214, -0.2473],\n",
       "          [-0.3859, -0.0526],\n",
       "          [-0.2567, -0.2440],\n",
       "          [-0.3577, -0.2431],\n",
       "          [-0.1498, -0.1054],\n",
       "          [-0.0944, -0.0710],\n",
       "          [-0.3146, -0.0387],\n",
       "          [-0.4252, -0.1438],\n",
       "          [-0.3226, -0.1421],\n",
       "          [-0.2960, -0.0817],\n",
       "          [-0.3211, -0.1310],\n",
       "          [-0.3853, -0.1161],\n",
       "          [-0.3109, -0.0364],\n",
       "          [-0.2977,  0.1462],\n",
       "          [-0.1328,  0.0314],\n",
       "          [-0.3342, -0.0701],\n",
       "          [-0.5700, -0.0115],\n",
       "          [-0.2519, -0.1362],\n",
       "          [-0.4578, -0.0894],\n",
       "          [-0.2698, -0.0551],\n",
       "          [-0.1669, -0.1093],\n",
       "          [-0.2553, -0.0846],\n",
       "          [-0.4316, -0.2538],\n",
       "          [-0.3316, -0.2551],\n",
       "          [-0.4073, -0.0751],\n",
       "          [-0.2202, -0.3173],\n",
       "          [-0.2794, -0.1389],\n",
       "          [-0.2425, -0.1792],\n",
       "          [-0.2832, -0.1098],\n",
       "          [-0.1287,  0.2229],\n",
       "          [-0.2639,  0.0712],\n",
       "          [-0.2201, -0.1134],\n",
       "          [-0.1492, -0.1659],\n",
       "          [-0.2776, -0.2189],\n",
       "          [-0.2878, -0.0415],\n",
       "          [-0.1805, -0.1663],\n",
       "          [-0.2792, -0.2314],\n",
       "          [-0.2126, -0.0437],\n",
       "          [-0.4348, -0.0335],\n",
       "          [-0.3286,  0.0007],\n",
       "          [-0.3617, -0.1236],\n",
       "          [-0.1425, -0.0521],\n",
       "          [-0.3000, -0.1547],\n",
       "          [-0.4410, -0.0949],\n",
       "          [-0.3183,  0.0522],\n",
       "          [-0.4971, -0.2710],\n",
       "          [-0.3785, -0.0070],\n",
       "          [-0.2178, -0.0191],\n",
       "          [-0.2380, -0.1480],\n",
       "          [-0.1239, -0.0281],\n",
       "          [-0.3794,  0.0738],\n",
       "          [-0.3396, -0.2411],\n",
       "          [-0.1765, -0.2005],\n",
       "          [-0.1802, -0.0525],\n",
       "          [-0.2632,  0.0024],\n",
       "          [-0.2948, -0.2185],\n",
       "          [-0.1290, -0.1136],\n",
       "          [-0.3083, -0.1619],\n",
       "          [-0.3293, -0.0767],\n",
       "          [-0.2807,  0.0439],\n",
       "          [-0.1912, -0.1623],\n",
       "          [-0.3985, -0.1207],\n",
       "          [-0.3332, -0.3964],\n",
       "          [-0.1821, -0.2357],\n",
       "          [-0.3507, -0.0833],\n",
       "          [-0.3261, -0.0887],\n",
       "          [-0.2686, -0.1020],\n",
       "          [-0.1585,  0.0811],\n",
       "          [-0.1133,  0.0327],\n",
       "          [-0.3969, -0.1683],\n",
       "          [-0.2950,  0.0459],\n",
       "          [-0.2956, -0.1088],\n",
       "          [-0.3274, -0.1936],\n",
       "          [-0.4770,  0.0076],\n",
       "          [-0.3784,  0.1040],\n",
       "          [-0.3905, -0.1402],\n",
       "          [-0.2577, -0.0544],\n",
       "          [-0.2279, -0.2493],\n",
       "          [-0.2616, -0.3007],\n",
       "          [-0.1924, -0.1371],\n",
       "          [-0.3829, -0.1344],\n",
       "          [-0.2907, -0.2312],\n",
       "          [-0.2120, -0.0679],\n",
       "          [-0.3637, -0.2723],\n",
       "          [-0.3366, -0.1155],\n",
       "          [-0.3082,  0.1035],\n",
       "          [-0.3053, -0.1084],\n",
       "          [-0.2945, -0.4320],\n",
       "          [-0.2166, -0.0466],\n",
       "          [-0.3556,  0.0612],\n",
       "          [-0.3389, -0.2033],\n",
       "          [-0.2612, -0.3451],\n",
       "          [-0.2395,  0.0049],\n",
       "          [-0.4306, -0.0493],\n",
       "          [-0.1829, -0.1374],\n",
       "          [-0.0960,  0.0523],\n",
       "          [-0.3800, -0.0580],\n",
       "          [-0.1712, -0.0389],\n",
       "          [-0.3478, -0.0915],\n",
       "          [-0.4035, -0.2092],\n",
       "          [-0.2322, -0.1390],\n",
       "          [-0.2613, -0.0754],\n",
       "          [-0.4354, -0.0729],\n",
       "          [-0.3875, -0.2475],\n",
       "          [-0.1923, -0.1714],\n",
       "          [-0.4316, -0.2538],\n",
       "          [-0.3786, -0.1762],\n",
       "          [-0.2601, -0.1861]]),\n",
       "  'reward': tensor([[ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.],\n",
       "          [-1.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [ 0.],\n",
       "          [-1.]]),\n",
       "  'next_state': tensor([[0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0325],\n",
       "          ...,\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 0.0000, 0.0000],\n",
       "          [0.0000, 0.0000, 3.9886, 4.0849]]),\n",
       "  'over': tensor([[1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [0],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [0],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [0],\n",
       "          [0],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0],\n",
       "          [0],\n",
       "          [1],\n",
       "          [1],\n",
       "          [1],\n",
       "          [0]])},\n",
       " -50.0)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class Data():\n",
    "    def __init__(self):\n",
    "        self.datas = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.datas)\n",
    "\n",
    "    def update(self):\n",
    "        state = env.reset()\n",
    "        over = False\n",
    "\n",
    "        data = {\n",
    "            'state': [],\n",
    "            'action': [],\n",
    "            'reward': [],\n",
    "            'next_state': [],\n",
    "            'over': [],\n",
    "        }\n",
    "\n",
    "        while not over:\n",
    "            action = ddpg.get_action(state)\n",
    "            next_state, reward, over = env.step(action)\n",
    "\n",
    "            data['state'].append(state)\n",
    "            data['action'].append(action)\n",
    "            data['reward'].append(reward)\n",
    "            data['next_state'].append(next_state)\n",
    "            data['over'].append(over)\n",
    "\n",
    "            state = next_state\n",
    "\n",
    "        self.datas.append(data)\n",
    "\n",
    "    def get_sample(self):\n",
    "        #采样结果\n",
    "        sample = {\n",
    "            'state': [],\n",
    "            'action': [],\n",
    "            'reward': [],\n",
    "            'next_state': [],\n",
    "            'over': [],\n",
    "        }\n",
    "\n",
    "        #采样N个数据\n",
    "        for _ in range(256):\n",
    "\n",
    "            #随机一局游戏\n",
    "            data = random.sample(self.datas, 1)[0]\n",
    "\n",
    "            #随机游戏中的一步,这里排除了最后一步\n",
    "            step = random.randint(0, len(data['action']) - 2)\n",
    "\n",
    "            #提取数据\n",
    "            state = data['state'][step]\n",
    "            next_state = data['next_state'][step]\n",
    "            action = data['action'][step]\n",
    "            reward = data['reward'][step]\n",
    "            over = data['over'][step]\n",
    "\n",
    "            #设置fake goal\n",
    "            if random.random() <= 0.8:\n",
    "\n",
    "                #随机选择step后面的某一步\n",
    "                step = random.randint(step + 1, len(data['action']) - 1)\n",
    "\n",
    "                #以后面某个步的状态为一个伪的终点,也就是希望先走到这里再说\n",
    "                fake_goal = data['state'][step][:2]\n",
    "\n",
    "                #求二范数\n",
    "                mod = [\n",
    "                    next_state[0] - fake_goal[0], next_state[1] - fake_goal[1]\n",
    "                ]\n",
    "                mod = torch.FloatTensor(mod).norm(p=2).item()\n",
    "\n",
    "                #再自己重新计算reward和over\n",
    "                reward = -1.0\n",
    "                over = False\n",
    "                if mod <= 0.15:\n",
    "                    reward = 0.0\n",
    "                    over = True\n",
    "\n",
    "                #以伪终点构建新的state\n",
    "                state[2] = fake_goal[0]\n",
    "                state[3] = fake_goal[1]\n",
    "                next_state[2] = fake_goal[0]\n",
    "                next_state[3] = fake_goal[1]\n",
    "\n",
    "            sample['state'].append(state)\n",
    "            sample['action'].append(action)\n",
    "            sample['reward'].append(reward)\n",
    "            sample['next_state'].append(next_state)\n",
    "            sample['over'].append(over)\n",
    "\n",
    "        sample['state'] = torch.FloatTensor(sample['state']).reshape(-1, 4)\n",
    "        sample['action'] = torch.FloatTensor(sample['action']).reshape(-1, 2)\n",
    "        sample['reward'] = torch.FloatTensor(sample['reward']).reshape(-1, 1)\n",
    "        sample['next_state'] = torch.FloatTensor(sample['next_state']).reshape(\n",
    "            -1, 4)\n",
    "        sample['over'] = torch.LongTensor(sample['over']).reshape(-1, 1)\n",
    "\n",
    "        return sample\n",
    "\n",
    "    #计算最后10个数据reward_sum的均值\n",
    "    def get_last_reward_mean(self):\n",
    "        reward_sum = []\n",
    "        for data in self.datas[-10:]:\n",
    "            reward_sum.append(sum(data['reward']))\n",
    "        return sum(reward_sum) / len(reward_sum)\n",
    "\n",
    "\n",
    "data = Data()\n",
    "\n",
    "#初始化数据\n",
    "for _ in range(200):\n",
    "    data.update()\n",
    "\n",
    "data.get_sample(), data.get_last_reward_mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 201 -50.0\n",
      "100 301 -45.3\n",
      "200 401 -50.0\n",
      "300 501 -50.0\n",
      "400 601 -23.1\n",
      "500 701 -4.6\n",
      "600 801 -4.2\n",
      "700 901 -5.6\n",
      "800 1001 -4.3\n",
      "900 1101 -3.9\n",
      "1000 1201 -4.3\n",
      "1100 1301 -4.5\n",
      "1200 1401 -4.2\n",
      "1300 1501 -4.3\n",
      "1400 1601 -4.1\n",
      "1500 1701 -4.4\n",
      "1600 1801 -4.1\n",
      "1700 1901 -4.8\n"
     ]
    }
   ],
   "source": [
    "for i in range(1800):\n",
    "    data.update()\n",
    "\n",
    "    for _ in range(20):\n",
    "        ddpg.train(**data.get_sample())\n",
    "\n",
    "    if i % 100 == 0:\n",
    "        print(i, len(data), data.get_last_reward_mean())"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "第19章-目标导向的强化学习.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
